{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in Tkinter callback\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\tkinter\\__init__.py\", line 1699, in __call__\n",
      "    return self.func(*args)\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\tkinter\\__init__.py\", line 745, in callit\n",
      "    func(*args)\n",
      "  File \"<ipython-input-1-54dd28abb728>\", line 138, in update\n",
      "    s, r, done = env.step(a)\n",
      "  File \"<ipython-input-1-54dd28abb728>\", line 93, in step\n",
      "    s = self.canvas.coords(self.rect)\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\tkinter\\__init__.py\", line 2463, in coords\n",
      "    self.tk.call((self._w, 'coords') + args))]\n",
      "_tkinter.TclError: invalid command name \".!canvas\"\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Reinforcement learning maze example.\n",
    "Red rectangle:          explorer.\n",
    "Black rectangles:       hells       [reward = -1].\n",
    "Yellow bin circle:      paradise    [reward = +1].\n",
    "All other states:       ground      [reward = 0].\n",
    "This script is the environment part of this example. The RL is in RL_brain.py.\n",
    "View more on my tutorial page: https://morvanzhou.github.io/tutorials/\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import time\n",
    "import sys\n",
    "if sys.version_info.major == 2:\n",
    "    import Tkinter as tk\n",
    "else:\n",
    "    import tkinter as tk\n",
    "\n",
    "\n",
    "UNIT = 40   # pixels\n",
    "MAZE_H = 4  # grid height\n",
    "MAZE_W = 4  # grid width\n",
    "\n",
    "\n",
    "class Maze(tk.Tk, object):\n",
    "    def __init__(self):\n",
    "        super(Maze, self).__init__()\n",
    "        self.action_space = ['u', 'd', 'l', 'r']\n",
    "        self.n_actions = len(self.action_space)\n",
    "        self.title('maze')\n",
    "        self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))\n",
    "        self._build_maze()\n",
    "\n",
    "    def _build_maze(self):\n",
    "        self.canvas = tk.Canvas(self, bg='white',\n",
    "                           height=MAZE_H * UNIT,\n",
    "                           width=MAZE_W * UNIT)\n",
    "\n",
    "        # create grids\n",
    "        for c in range(0, MAZE_W * UNIT, UNIT):\n",
    "            x0, y0, x1, y1 = c, 0, c, MAZE_H * UNIT\n",
    "            self.canvas.create_line(x0, y0, x1, y1)\n",
    "        for r in range(0, MAZE_H * UNIT, UNIT):\n",
    "            x0, y0, x1, y1 = 0, r, MAZE_W * UNIT, r\n",
    "            self.canvas.create_line(x0, y0, x1, y1)\n",
    "\n",
    "        # create origin\n",
    "        origin = np.array([20, 20])\n",
    "\n",
    "        # hell\n",
    "        hell1_center = origin + np.array([UNIT * 2, UNIT])\n",
    "        self.hell1 = self.canvas.create_rectangle(\n",
    "            hell1_center[0] - 15, hell1_center[1] - 15,\n",
    "            hell1_center[0] + 15, hell1_center[1] + 15,\n",
    "            fill='black')\n",
    "        # hell\n",
    "        hell2_center = origin + np.array([UNIT, UNIT * 2])\n",
    "        self.hell2 = self.canvas.create_rectangle(\n",
    "            hell2_center[0] - 15, hell2_center[1] - 15,\n",
    "            hell2_center[0] + 15, hell2_center[1] + 15,\n",
    "            fill='black')\n",
    "\n",
    "        # create oval\n",
    "        oval_center = origin + UNIT * 2\n",
    "        self.oval = self.canvas.create_oval(\n",
    "            oval_center[0] - 15, oval_center[1] - 15,\n",
    "            oval_center[0] + 15, oval_center[1] + 15,\n",
    "            fill='yellow')\n",
    "\n",
    "        # create red rect\n",
    "        self.rect = self.canvas.create_rectangle(\n",
    "            origin[0] - 15, origin[1] - 15,\n",
    "            origin[0] + 15, origin[1] + 15,\n",
    "            fill='red')\n",
    "\n",
    "        # pack all\n",
    "        self.canvas.pack()\n",
    "\n",
    "    def reset(self):\n",
    "        self.update()\n",
    "        time.sleep(0.5)\n",
    "        self.canvas.delete(self.rect)\n",
    "        origin = np.array([20, 20])\n",
    "        self.rect = self.canvas.create_rectangle(\n",
    "            origin[0] - 15, origin[1] - 15,\n",
    "            origin[0] + 15, origin[1] + 15,\n",
    "            fill='red')\n",
    "        # return observation\n",
    "        return self.canvas.coords(self.rect)\n",
    "\n",
    "    def step(self, action):\n",
    "        s = self.canvas.coords(self.rect)\n",
    "        base_action = np.array([0, 0])\n",
    "        if action == 0:   # up\n",
    "            if s[1] > UNIT:\n",
    "                base_action[1] -= UNIT\n",
    "        elif action == 1:   # down\n",
    "            if s[1] < (MAZE_H - 1) * UNIT:\n",
    "                base_action[1] += UNIT\n",
    "        elif action == 2:   # right\n",
    "            if s[0] < (MAZE_W - 1) * UNIT:\n",
    "                base_action[0] += UNIT\n",
    "        elif action == 3:   # left\n",
    "            if s[0] > UNIT:\n",
    "                base_action[0] -= UNIT\n",
    "\n",
    "        self.canvas.move(self.rect, base_action[0], base_action[1])  # move agent\n",
    "\n",
    "        s_ = self.canvas.coords(self.rect)  # next state\n",
    "\n",
    "        # reward function\n",
    "        if s_ == self.canvas.coords(self.oval):\n",
    "            reward = 1\n",
    "            done = True\n",
    "            s_ = 'terminal'\n",
    "        elif s_ in [self.canvas.coords(self.hell1), self.canvas.coords(self.hell2)]:\n",
    "            reward = -1\n",
    "            done = True\n",
    "            s_ = 'terminal'\n",
    "        else:\n",
    "            reward = 0\n",
    "            done = False\n",
    "\n",
    "        return s_, reward, done\n",
    "\n",
    "    def render(self):\n",
    "        time.sleep(0.1)\n",
    "        self.update()\n",
    "\n",
    "\n",
    "def update():\n",
    "    for t in range(10):\n",
    "        s = env.reset()\n",
    "        while True:\n",
    "            env.render()\n",
    "            a = 1\n",
    "            s, r, done = env.step(a)\n",
    "            if done:\n",
    "                break\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    env = Maze()\n",
    "    env.after(100, update)\n",
    "    env.mainloop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
